{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "9e50a1cf-811c-4e1b-94e4-86355b4def29",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "# from sklearn.model_selection import train_test_split\n",
    "import tensorflow as tf\n",
    "import os\n",
    "import sys\n",
    "from tensorflow.keras.layers import *\n",
    "import tensorflow.keras.backend as K\n",
    "from tqdm import tqdm\n",
    "from feature_utils import *\n",
    "import argparse\n",
    "import datetime"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "93ec1696-8f0a-4a37-a590-44c699f34106",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_dataset():\n",
    "    dfiles = os.listdir('out/')\n",
    "    absfiles = [\"out/\" + f for f in dfiles]\n",
    "    dfs = []\n",
    "    for af in absfiles:\n",
    "        df = pd.read_csv(af, names = COLUMNS, header = None)\n",
    "        dfs.append(df)\n",
    "    df = pd.concat(dfs)\n",
    "    df['dt'] = pd.to_datetime(df.date)\n",
    "    return df\n",
    "\n",
    "def gen_input(X):\n",
    "    Y = X.loc[:, 'target']\n",
    "    return X.loc[:, FNAMES].values, Y.values\n",
    "\n",
    "def my_loss_fn(y_true, y_pred):\n",
    "    loss = K.binary_crossentropy(y_true, y_pred)\n",
    "    #weight_vector = y_true * 1.3 + (1 - y_true) * 0.3\n",
    "    #wloss = weight_vector * loss\n",
    "    wloss = loss\n",
    "    return K.mean(wloss)\n",
    "\n",
    "def model_fun():\n",
    "    cinput_map = {}\n",
    "    emb_list = []\n",
    "    X = tf.keras.Input(len(FNAMES), name = 'X')\n",
    "    inputs = [X]\n",
    "    dnn_part = Dense(64, activation='relu')(X)\n",
    "    dnn_part = tf.keras.layers.BatchNormalization()(dnn_part)\n",
    "    last = dnn_part\n",
    "    last = tf.keras.layers.BatchNormalization()(last)\n",
    "    last = tf.keras.layers.Dense(64, activation='relu')(last)\n",
    "    last = tf.keras.layers.BatchNormalization()(last)\n",
    "    final = tf.keras.layers.Dense(1, activation='sigmoid')(last)\n",
    "    model = tf.keras.Model(inputs = inputs, outputs = final)\n",
    "\n",
    "    model.compile(\n",
    "        loss='binary_crossentropy',\n",
    "        #loss=my_loss_fn,\n",
    "        optimizer='adam',\n",
    "        metrics=['accuracy', tf.keras.metrics.AUC()])\n",
    "    return model\n",
    "\n",
    "\n",
    "def flag_fun(row):\n",
    "    if row['target'] == 1.0 and row['Y'] > 0.7:\n",
    "        return 0\n",
    "    elif row['target'] == 0.0 and row['Y'] < 0.3:\n",
    "        return 1\n",
    "    elif (row['target'] == 0.0 and row['Y'] > 0.7) or (row['target'] == 1.0 and row['Y'] < 0.3):\n",
    "        return 2\n",
    "    else:\n",
    "        return 3\n",
    "\n",
    "\n",
    "\n",
    "# model_dir = \"dmodel\"\n",
    "# if ARGS.job == 'train':\n",
    "#     callbacks = []\n",
    "#     log_dir = \"train_logs/\"\n",
    "#     tensorboard_callback = tf.keras.callbacks.TensorBoard(log_dir=log_dir, histogram_freq=1)\n",
    "#     callbacks.append(tensorboard_callback)\n",
    "#     model = model_fun()\n",
    "#     model.fit(X_train, Y_train, epochs = 3, callbacks=callbacks)\n",
    "#     model.save(model_dir)\n",
    "#     model.evaluate(X_test, Y_test)\n",
    "# elif ARGS.job == 'eval':\n",
    "#     model = tf.keras.models.load_model(model_dir)\n",
    "#     model.evaluate(X_test, Y_test)\n",
    "# elif ARGS.job == 'predict':\n",
    "#     model = tf.keras.models.load_model(model_dir)\n",
    "#     Y_res = model.predict(X_test)\n",
    "#     X2.loc[:, \"target\"] = Y_test\n",
    "#     X2.loc[:, \"Y\"] = Y_res\n",
    "#     X2.loc[:, \"FLAG\"] = X2.apply(flag_fun, axis=1)\n",
    "#     for i in range(4):\n",
    "#         X2[X2.FLAG == i].to_csv(\"/tmp/p_%d.csv\" % i, index = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "d26fd485-2510-4bc3-b017-43d0712b15d1",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = get_dataset()\n",
    "X = df[df.dt < '2019-01-01']\n",
    "X2 = df[df.dt >= '2019-01-01']\n",
    "\n",
    "N_NUM_FEATURES = len(FNAMES)\n",
    "# N_CAT_FEATURES = len(CAT_FNAMES)\n",
    "\n",
    "X_train, Y_train = gen_input(X)\n",
    "X_test, Y_test = gen_input(X2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "30a20720-174a-4868-b669-bb4bcfb25da5",
   "metadata": {},
   "outputs": [],
   "source": [
    "# model_dir = \"model\"\n",
    "# if ARGS.job == 'train':\n",
    "#     callbacks = []\n",
    "#     log_dir = \"train_logs/\"\n",
    "#     tensorboard_callback = tf.keras.callbacks.TensorBoard(log_dir=log_dir, histogram_freq=1)\n",
    "#     callbacks.append(tensorboard_callback)\n",
    "#     model = model_fun()\n",
    "#     model.fit(X_train, Y_train, epochs = 10, callbacks=callbacks)\n",
    "#     model.save(model_dir)\n",
    "#     model.evaluate(X_test, Y_test)\n",
    "# elif ARGS.job == 'eval':\n",
    "#     model = tf.keras.models.load_model(model_dir)\n",
    "#     model.evaluate(X_test, Y_test)\n",
    "# elif ARGS.job == 'predict':\n",
    "#     model = tf.keras.models.load_model(model_dir)\n",
    "#     Y_res = model.predict(X_test)\n",
    "#     X2[\"target\"] = Y_test\n",
    "#     X2[\"Y\"] = Y_res\n",
    "#     X2[\"FLAG\"] = X2.apply(flag_fun, axis=1)\n",
    "#     for i in range(4):\n",
    "#         X2[X2.FLAG == i].to_csv(\"/tmp/p_%d_csv\" % i, index = False)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "52496114-b9a7-40c0-8e79-e5dd78ab8f3c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/10\n",
      "5434/5434 [==============================] - 31s 5ms/step - loss: 0.2962 - accuracy: 0.8754 - auc: 0.8217\n",
      "Epoch 2/10\n",
      "5434/5434 [==============================] - 27s 5ms/step - loss: 0.2792 - accuracy: 0.8801 - auc: 0.8466\n",
      "Epoch 3/10\n",
      "5434/5434 [==============================] - 25s 5ms/step - loss: 0.2751 - accuracy: 0.8817 - auc: 0.8523\n",
      "Epoch 4/10\n",
      "5434/5434 [==============================] - 25s 5ms/step - loss: 0.2731 - accuracy: 0.8827 - auc: 0.8550\n",
      "Epoch 5/10\n",
      "5434/5434 [==============================] - 26s 5ms/step - loss: 0.2713 - accuracy: 0.8829 - auc: 0.8577\n",
      "Epoch 6/10\n",
      "5434/5434 [==============================] - 23s 4ms/step - loss: 0.2697 - accuracy: 0.8835 - auc: 0.8599\n",
      "Epoch 7/10\n",
      "5434/5434 [==============================] - 24s 4ms/step - loss: 0.2690 - accuracy: 0.8832 - auc: 0.8613\n",
      "Epoch 8/10\n",
      "5434/5434 [==============================] - 24s 5ms/step - loss: 0.2673 - accuracy: 0.8837 - auc: 0.8635\n",
      "Epoch 9/10\n",
      "5434/5434 [==============================] - 24s 4ms/step - loss: 0.2675 - accuracy: 0.8841 - auc: 0.8629\n",
      "Epoch 10/10\n",
      "5434/5434 [==============================] - 24s 4ms/step - loss: 0.2678 - accuracy: 0.8837 - auc: 0.8628\n"
     ]
    },
    {
     "ename": "NameError",
     "evalue": "name 'model_dir' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m\u001b[0m",
      "\u001b[0;31mNameError\u001b[0mTraceback (most recent call last)",
      "\u001b[0;32m<ipython-input-6-a114cbd66fe8>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0mmodel\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmodel_fun\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      6\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX_train\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mY_train\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mepochs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;36m10\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcallbacks\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcallbacks\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 7\u001b[0;31m \u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msave\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel_dir\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      8\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mevaluate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX_test\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mY_test\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mNameError\u001b[0m: name 'model_dir' is not defined"
     ]
    }
   ],
   "source": [
    "callbacks = []\n",
    "log_dir = \"train_logs/\"\n",
    "tensorboard_callback = tf.keras.callbacks.TensorBoard(log_dir=log_dir, histogram_freq=1)\n",
    "callbacks.append(tensorboard_callback)\n",
    "model = model_fun()\n",
    "model.fit(X_train, Y_train, epochs = 10, callbacks=callbacks)\n",
    "model.save(model_dir)\n",
    "model.evaluate(X_test, Y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "13d9eaeb-f504-40b8-9960-12f710741bf1",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
